﻿//  Symplectic Spark
//  Copyright 2010 Symplectic Ltd
//  Created by Martyn Whitwell (martyn@symplectic.co.uk)

//  This file is part of Spark.

//  Spark is free software: you can redistribute it and/or modify
//  it under the terms of the GNU General Public License as published by
//  the Free Software Foundation, either version 3 of the License, or
//  (at your option) any later version.

//  Spark is distributed in the hope that it will be useful,
//  but WITHOUT ANY WARRANTY; without even the implied warranty of
//  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
//  GNU General Public License for more details.

//  You should have received a copy of the GNU General Public License
//  along with Spark.  If not, see <http://www.gnu.org/licenses/>.

// $URL: https://symplectic-spark.googlecode.com/svn/trunk/Symplectic.Spark.Rebuffer/Program.cs $
// $LastChangedDate: 2010-06-09 11:38:27 +0100 (Wed, 09 Jun 2010) $
// $LastChangedRevision: 52 $
// $LastChangedBy: martyn@symplectic.co.uk $

using System;
using System.Configuration;
using System.Text;
using System.IO;
using System.Linq;
using System.Xml;
using System.Xml.Linq;
using System.Xml.XPath;
using System.Diagnostics;
using System.Collections;

using log4net;

namespace Symplectic.Spark
{
    class Program
    {
        #region Logger
        private static readonly log4net.ILog Logger = log4net.LogManager.GetLogger(System.Reflection.MethodBase.GetCurrentMethod().DeclaringType);
        #endregion

        #region Configuration Settings
        private static readonly string apiBaseUrl = ConfigurationSettings.AppSettings["api-base-url"];
        private static readonly int apiInterval = int.Parse(ConfigurationSettings.AppSettings["api-interval"]);
        private static readonly string componentsFolder = ConfigurationSettings.AppSettings["components-folder"];
        private static readonly string bufferFolder = ConfigurationSettings.AppSettings["buffer-folder"];
        private static readonly string bufferSettingsFile = Path.Combine(bufferFolder, "buffer-settings.xml");
        private static readonly int timeout = int.Parse(ConfigurationSettings.AppSettings["timeout"]);
        private static readonly int apiRetryMaxCount = int.Parse(ConfigurationSettings.AppSettings["api-retry-max-count"]);
        private static readonly int fileRetryMaxCount = int.Parse(ConfigurationSettings.AppSettings["file-retry-max-count"]);
        #endregion

        private enum Mode { Differential, Full }

        static void Main(string[] args)
        {
            Stopwatch stopwatch = Stopwatch.StartNew();            

            Mode mode;
            if (args.Length == 1)
                switch (args[0].ToLower().Trim())
                {
                    case "full":
                        mode = Mode.Full;
                        break;
                    case "differential":
                    default:
                        mode = Mode.Differential;
                        break;
                }
            else
                mode = Mode.Differential;
            
            Logger.InfoFormat("Buffering data");
            Logger.InfoFormat("Buffering mode: {0}", mode);

            Logger.DebugFormat("API:\t{0}", apiBaseUrl);
            Logger.DebugFormat("API Interval:\t{0}", apiInterval);
            Logger.DebugFormat("Buffer Folder:\t{0}", bufferFolder);
            Logger.DebugFormat("Components Folder:\t{0}", componentsFolder);
            Logger.DebugFormat("Timeout:\t{0}", timeout);
            Logger.DebugFormat("API RetryMaxCount:\t{0}", apiRetryMaxCount);
            Logger.DebugFormat("File RetryMaxCount:\t{0}", fileRetryMaxCount);

            Buffer buffer = new Buffer(bufferFolder, fileRetryMaxCount, apiBaseUrl, apiInterval, timeout, apiRetryMaxCount);
            int errorCount = 0;

            int count = 0;
            foreach (ConfigBuffer configBuffer in ConfigBufferSettings.Config.Buffers)
            {
                try
                {
                    //setup variables
                    DateTime starttime = DateTime.Now;
                    XElement items = new XElement("items");
                    string modifiedSinceKey = configBuffer.Name + "-modified-since";
                    string uri = AppendModifiedSince(configBuffer.SourceItems.ApiRelativeUri, (configBuffer.SourceItems.ApiUseModifiedSince && mode == Mode.Differential), ReadBufferSetting(modifiedSinceKey));
                    int processCount = 0;

                    //load list of items to buffer
                    int sourceItemsCount = buffer.ReadPagedData(configBuffer.SourceItems.SelectElements, uri, configBuffer.SourceItems.ApiPerPage, items);
                    Logger.DebugFormat("Loaded {0} {1} items: {2}", configBuffer.Name, uri, sourceItemsCount);

                    //Does each item need buffering? if so, call buffer.process()
                    if (configBuffer.Settings.BufferMode == ConfigBufferBufferSettings.enBufferMode.BufferItem)
                    {
                        //Buffer the items, one at a time
                        processCount += buffer.BufferItems(items, configBuffer.BufferItem);
                        Logger.DebugFormat("Buffered {0} {1} records: {2}", configBuffer.Name, uri, processCount);
                    }

                    //Else, just write the items straight to disk
                    if (configBuffer.Settings.BufferMode == ConfigBufferBufferSettings.enBufferMode.WriteItems)
                    {
                        //Write the data
                        processCount += buffer.WriteItems(items, configBuffer.WriteItems);
                        Logger.DebugFormat("Buffered {0} {1} records: {2}", configBuffer.Name, uri, processCount);
                    }

                    WriteBufferSetting(modifiedSinceKey, starttime.ToString("o"));
                    count += processCount;
                }
                catch (Exception ex)
                {
                    Logger.ErrorFormat("Failed to buffer {0}: {1}", configBuffer.Name, ex);
                    errorCount++;
                }
            }

            stopwatch.Stop();

            Logger.InfoFormat("All done, processed {0} objects in {1} seconds ({2} objects/second)", count, (int)stopwatch.Elapsed.TotalSeconds, (int)(count / stopwatch.Elapsed.TotalSeconds));

            if (errorCount > 0)
                Logger.ErrorFormat("There were {0} errors in processing {1} configuration(s). THESE SHOULD BE INVESTIGATED IMMEDIATELY.", errorCount, ConfigBufferSettings.Config.Buffers.Count);
            
            System.Environment.ExitCode = errorCount;

            Console.ReadLine();
        }


        private static XDocument ReadBufferSettings()
        {
            //Create new XmlDocument
            XDocument bufferSettings = null;

            if (File.Exists(bufferSettingsFile))
            {
                try
                {
                    using (FileStream fs = File.Open(bufferSettingsFile, FileMode.Open, FileAccess.Read, FileShare.ReadWrite))
                    using (XmlTextReader xtr = new XmlTextReader(fs))
                    {
                        bufferSettings = XDocument.Load(xtr, LoadOptions.None);
                    }
                }
                catch (XmlException exc)
                {
                    Logger.ErrorFormat("{0}: Buffer settings file could not be loaded and has been corrupted: {1}", exc.Message, bufferSettingsFile);
                }
                catch (IOException exc)
                {
                    Logger.WarnFormat("{0}: Buffer settings file could not be read: {1}", exc.Message, bufferSettingsFile);
                }
            }
            else
                Logger.WarnFormat("Buffer settings file does not exist: {0}", bufferSettingsFile);

            //If bufferSettings could not be initialised, create a new one
            if (bufferSettings == null)
            {
                bufferSettings = new XDocument();
                bufferSettings.Add(new XElement("buffer-settings"));
            }

            return bufferSettings;
        }


        private static string AppendModifiedSince(string apiRelativeUri, bool useModifiedSince, string modifiedSince)
        {
            if (useModifiedSince && modifiedSince != null)
            {
                if (apiRelativeUri.Contains("?"))
                    return apiRelativeUri + "&modified-since=" + DateTime.Parse(modifiedSince).ToUniversalTime().ToString("o");
                else
                    return apiRelativeUri + "?modified-since=" + DateTime.Parse(modifiedSince).ToUniversalTime().ToString("o");
            }
            else return apiRelativeUri;
        }

        
        private static string ReadBufferSetting(string key)
        {
            XDocument bufferSettings = ReadBufferSettings();
            XElement setting = bufferSettings.Root.Element(key);
            return setting == null ? null : setting.Value;
        }

        private static bool WriteBufferSetting(string key, string value)
        {
            XDocument bufferSettings = ReadBufferSettings();
            XElement setting = bufferSettings.Root.Element(key);
            bool success = false;

            if (setting != null)
                setting.Value = value;
            else
            {
                setting = new XElement(key, value);
                bufferSettings.Root.Add(setting);
            }

            try
            {
                using (FileStream fs = File.Open(bufferSettingsFile, FileMode.Create, FileAccess.Write, FileShare.Read))
                using (StreamWriter sw = new StreamWriter(fs, Encoding.UTF8))
                {
                    bufferSettings.Save(sw, SaveOptions.None);
                    success = true;
                }
            }
            catch (IOException exc)
            {
                Logger.WarnFormat("{0}: Buffer settings file could not be writen to: {1}", exc.Message, bufferSettingsFile);
            }

            return success;
        }
    }
}